From: Petr Štetiar Date: Tue, 16 Mar 2021 17:21:15 +0000 (+0100) Subject: phase1,phase2: implement round robin builds X-Git-Tag: v1~26 X-Git-Url: http://git.openwrt.org/%22https:/collectd.org/%22http:/www.crowdsec.net//%22https%22/%22https:/collectd.org/%22http:/www.crowdsec.net/%22https%22?a=commitdiff_plain;h=27f0b7fe926d70c4f3e9c5bb1b297b76d6e1f704;p=buildbot.git phase1,phase2: implement round robin builds Gather newest complete and not skipped build timestamps, reverse sort them and use that as builder priority, so the newest built targets are at the bottom. Signed-off-by: Petr Štetiar --- diff --git a/phase1/master.cfg b/phase1/master.cfg index 2ccac8e..53d1812 100644 --- a/phase1/master.cfg +++ b/phase1/master.cfg @@ -4,13 +4,17 @@ import os import re import base64 -import random import subprocess import configparser -from datetime import timedelta +from dateutil.tz import tzutc +from datetime import datetime, timedelta + +from twisted.internet import defer +from twisted.python import log from buildbot import locks +from buildbot.data import resultspec from buildbot.changes import filter from buildbot.changes.gitpoller import GitPoller from buildbot.config import BuilderConfig @@ -19,6 +23,7 @@ from buildbot.plugins import schedulers from buildbot.plugins import steps from buildbot.plugins import util from buildbot.process import properties +from buildbot.process import results from buildbot.process.factory import BuildFactory from buildbot.process.properties import Interpolate from buildbot.process.properties import Property @@ -123,6 +128,69 @@ c['configurators'] = [util.JanitorConfigurator( hour=6, )] +@defer.inlineCallbacks +def getNewestCompleteTime(bldr): + """Returns the complete_at of the latest completed and not SKIPPED + build request for this builder, or None if there are no such build + requests. We need to filter out SKIPPED requests because we're + using collapseRequests=True which is unfortunately marking all + previous requests as complete when new buildset is created. + + @returns: datetime instance or None, via Deferred + """ + + bldrid = yield bldr.getBuilderId() + completed = yield bldr.master.data.get( + ('builders', bldrid, 'buildrequests'), + [ + resultspec.Filter('complete', 'eq', [True]), + resultspec.Filter('results', 'ne', [results.SKIPPED]), + ], + order=['-complete_at'], limit=1) + if not completed: + return + + return completed[0]['complete_at'] + +@defer.inlineCallbacks +def prioritizeBuilders(master, builders): + """Returns sorted list of builders by their last timestamp of completed and + not skipped build. + + @returns: list of sorted builders + """ + + def is_building(bldr): + return bool(bldr.building) or bool(bldr.old_building) + + def bldr_info(bldr): + d = defer.maybeDeferred(getNewestCompleteTime, bldr) + d.addCallback(lambda complete_at: (complete_at, bldr)) + return d + + def bldr_sort(item): + (complete_at, bldr) = item + + if not complete_at: + date = datetime.min + complete_at = date.replace(tzinfo=tzutc()) + + if is_building(bldr): + date = datetime.max + complete_at = date.replace(tzinfo=tzutc()) + + return (complete_at, bldr.name) + + results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders]) + results.sort(key=bldr_sort) + + for r in results: + log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0])) + + return [r[1] for r in results] + +c['prioritizeBuilders'] = prioritizeBuilders + ####### CHANGESOURCES work_dir = os.path.abspath(ini.get("general", "workdir") or ".") @@ -456,13 +524,10 @@ def GetNextBuild(builder, requests): for r in requests: if r.properties and r.properties.hasProperty("tag"): return r - return requests[0] -def prioritizeBuilders(buildmaster, builders): - random.shuffle(builders) - return builders - -c['prioritizeBuilders'] = prioritizeBuilders + r = requests[0] + log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid)) + return r def MakeEnv(overrides=None, tryccache=False): env = { diff --git a/phase2/master.cfg b/phase2/master.cfg index ac44fcd..a54c191 100644 --- a/phase2/master.cfg +++ b/phase2/master.cfg @@ -8,15 +8,21 @@ import random import subprocess import configparser -from datetime import timedelta +from dateutil.tz import tzutc +from datetime import datetime, timedelta + +from twisted.internet import defer +from twisted.python import log from buildbot import locks +from buildbot.data import resultspec from buildbot.changes import filter from buildbot.changes.gitpoller import GitPoller from buildbot.config import BuilderConfig from buildbot.plugins import schedulers from buildbot.plugins import steps from buildbot.plugins import util +from buildbot.process import results from buildbot.process.factory import BuildFactory from buildbot.process.properties import Property from buildbot.process.properties import WithProperties @@ -319,9 +325,66 @@ def UsignSec2Pub(seckey, comment="untrusted comment: secret key"): def IsSharedWorkdir(step): return bool(step.getProperty("shared_wd")) -def prioritizeBuilders(buildmaster, builders): - random.shuffle(builders) - return builders +@defer.inlineCallbacks +def getNewestCompleteTime(bldr): + """Returns the complete_at of the latest completed and not SKIPPED + build request for this builder, or None if there are no such build + requests. We need to filter out SKIPPED requests because we're + using collapseRequests=True which is unfortunately marking all + previous requests as complete when new buildset is created. + + @returns: datetime instance or None, via Deferred + """ + + bldrid = yield bldr.getBuilderId() + completed = yield bldr.master.data.get( + ('builders', bldrid, 'buildrequests'), + [ + resultspec.Filter('complete', 'eq', [True]), + resultspec.Filter('results', 'ne', [results.SKIPPED]), + ], + order=['-complete_at'], limit=1) + if not completed: + return + + return completed[0]['complete_at'] + +@defer.inlineCallbacks +def prioritizeBuilders(master, builders): + """Returns sorted list of builders by their last timestamp of completed and + not skipped build. + + @returns: list of sorted builders + """ + + def is_building(bldr): + return bool(bldr.building) or bool(bldr.old_building) + + def bldr_info(bldr): + d = defer.maybeDeferred(getNewestCompleteTime, bldr) + d.addCallback(lambda complete_at: (complete_at, bldr)) + return d + + def bldr_sort(item): + (complete_at, bldr) = item + + if not complete_at: + date = datetime.min + complete_at = date.replace(tzinfo=tzutc()) + + if is_building(bldr): + date = datetime.max + complete_at = date.replace(tzinfo=tzutc()) + + return (complete_at, bldr.name) + + results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders]) + results.sort(key=bldr_sort) + + for r in results: + log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0])) + + return [r[1] for r in results] c['prioritizeBuilders'] = prioritizeBuilders c['builders'] = []